{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gensim in c:\\programdata\\anaconda3\\lib\\site-packages (4.2.0)\n",
      "Requirement already satisfied: numpy>=1.17.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from gensim) (1.21.6)\n",
      "Requirement already satisfied: scipy>=0.18.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from gensim) (1.1.0)\n",
      "Requirement already satisfied: smart-open>=1.8.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from gensim) (7.1.0)\n",
      "Requirement already satisfied: Cython==0.29.28 in c:\\programdata\\anaconda3\\lib\\site-packages (from gensim) (0.29.28)\n",
      "Requirement already satisfied: wrapt in c:\\programdata\\anaconda3\\lib\\site-packages (from smart-open>=1.8.1->gensim) (1.10.11)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DEPRECATION: pandas 0.23.4 has a non-standard dependency specifier pytz>=2011k. pip 24.1 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of pandas or contact the author to suggest that they release a version with a conforming dependency specifiers. Discussion can be found at https://github.com/pypa/pip/issues/12063\n"
     ]
    }
   ],
   "source": [
    "!pip install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "‘周瑜’的词向量为:\n",
      " [-0.1061348  -0.06395987  0.24998227  1.0366884   0.22929919  0.04740789\n",
      "  0.09813576  0.06329137 -0.8477018   0.1561368   0.70274913 -0.950956\n",
      " -0.37306803 -0.36454442  0.06094892  0.3204792  -0.23562004  0.1401825\n",
      " -0.51827073 -1.4115244 ]\n",
      "与‘周瑜’相似度最高的10个词:\n",
      "[('孙策', 0.922085165977478), ('袁绍', 0.9057745933532715), ('袁术', 0.9056820273399353), ('邓艾', 0.9054704904556274), ('吕布', 0.9050405621528625), ('陆逊', 0.9046210050582886), ('钟会', 0.9044021964073181), ('孟获', 0.9020810127258301), ('孔明', 0.9006070494651794), ('曹真', 0.8914230465888977)]\n",
      "‘刘备’和‘曹操’的相似度:0.8142221570014954\n",
      "在词'孙权 /曹操 /刘备 /孙夫人'中，‘孙夫人’与其他此不属于同一类\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "import re\n",
    "from gensim.models import Word2Vec\n",
    "with open(r\"sanguo.txt\",encoding='utf-8')as f:\n",
    "    lines=[]\n",
    "    for line in f:\n",
    "        temp=jieba.lcut(line)\n",
    "        words=[]\n",
    "        for i in temp:\n",
    "            i = re.sub(\"[\\s+\\.\\!\\/_,$%^*(+\\\"\\'””《》]+|[+——！，。？、~@#￥%……&*（）：；‘]+\", \"\", i)\n",
    "            if len(i)>0:\n",
    "                words.append(i)\n",
    "        if len(words)>0:\n",
    "            lines.append(words)\n",
    "model=Word2Vec(lines,vector_size=20,window=2,min_count=3,epochs=7,negative=10,sg=1)\n",
    "print(\"‘周瑜’的词向量为:\\n\",model.wv.get_vector('周瑜'))\n",
    "print(\"与‘周瑜’相似度最高的10个词:\")\n",
    "print(model.wv.most_similar('周瑜',topn=10))\n",
    "print(\"‘刘备’和‘曹操’的相似度:{}\".format(model.wv.similarity('刘备','曹操')))\n",
    "words=\"孙权 曹操 刘备 孙夫人\"\n",
    "print(\"在词'孙权 /曹操 /刘备 /孙夫人'中，‘{}’与其他此不属于同一类\".format(model.wv.doesnt_match(words.split())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "from gensim.models.doc2vec import Doc2Vec\n",
    "TaggededDocument=gensim.models.doc2vec.TaggedDocument\n",
    "def get_data():\n",
    "    with open(\"train.txt\",'r',encoding='utf-8')as f:\n",
    "        docs=f.readlines()\n",
    "    train_data=[]\n",
    "    for i,text in enumerate(docs):\n",
    "        word_list=text.split(' ')\n",
    "        word_list[len(word_list)-1]=word_list[len(word_list)-1].strip()\n",
    "        document=TaggededDocument(word_list,tags=[i])\n",
    "        train_data.append(document)\n",
    "    return train_data\n",
    "def train_model(x_train):\n",
    "    model_dm=Doc2Vec(x_train,min_count=1,window=3,vector_size=20,negative=5,workers=4,dm=1) \n",
    "    model_dm.train(x_train,total_examples=model_dm.corpus_count,epochs=70)\n",
    "    model_dm.save(\"model_doc2vec\")\n",
    "    return model_dm\n",
    "def test():\n",
    "    model_dm=Doc2Vec.load('model_doc2vec')\n",
    "    test_text=['科学','教育','是','难搞','的']\n",
    "    inferred_vector_dm=model_dm.infer_vector(test_text)\n",
    "    sims=model_dm.docvecs.most_similar([inferred_vector_dm],topn=10)\n",
    "    return sims\n",
    "if __name__ == '__main__':\n",
    "    train_data=get_data()\n",
    "    model_dm=train_model(train_data)\n",
    "    sims=test()\n",
    "    print('相似文本，相似度和文本中词的数量：')\n",
    "    for count,sim in sims:\n",
    "        sentence=train_data[count]\n",
    "        words=''\n",
    "        for word in sentence[0]:\n",
    "            words=words+word+' '\n",
    "        print(words,sim,len(sentence[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
